package streaming.demo.bigdata.hdfs;

import org.apache.commons.lang.SystemUtils;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.serialization.SimpleStringEncoder;
import org.apache.flink.core.fs.Path;
import org.apache.flink.runtime.state.filesystem.FsStateBackend;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.filesystem.OutputFileConfig;
import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink;
import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.DefaultRollingPolicy;
import utils.PropertiesReader;

import java.util.concurrent.TimeUnit;

/**
 *  StreamingFileSink
 */
public class SinkHDFSTest1 {

    private static String socketIP = PropertiesReader.get("default.socket.ip");
    private static Integer socketPort = PropertiesReader.getInteger("default.socket.port");
    private static String HDFSRoot = PropertiesReader.get("default.file.hdfs.url");
    private static String toPathDir = PropertiesReader.get("default.file.hdfs.basePath");

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
        env.setParallelism(1);

        env.enableCheckpointing(1000L);
        if(SystemUtils.IS_OS_WINDOWS) {
            env.setStateBackend(new FsStateBackend("file:///E:/testData/ckp"));
        } else {
            env.setStateBackend(new FsStateBackend(HDFSRoot + "/flink-checkpoint"));
        }

        DataStreamSource<String> dataStream = env.socketTextStream(socketIP, socketPort);

        dataStream.print();



        OutputFileConfig config = OutputFileConfig.builder()
//                .withPartPrefix("zzb")
                .withPartSuffix(".txt")
                .build();

        StreamingFileSink<String> streamingFileSink = StreamingFileSink.forRowFormat(
                new Path(HDFSRoot + toPathDir),
                new SimpleStringEncoder<String>("UTF-8")
        ).withRollingPolicy(
                DefaultRollingPolicy.builder()
                        .withRolloverInterval(TimeUnit.MINUTES.toMillis(15)) //每隔15分钟生成一个新文件
                        .withInactivityInterval(TimeUnit.MINUTES.toMillis(5)) //每隔5分钟没有新数据到来，也把之前的生成一个新文件
                        .withMaxPartSize(1024 * 1024 * 1024L)
                        .build()
        )
                .withOutputFileConfig(config)
                .build();

        dataStream.addSink(streamingFileSink);

        env.execute();
    }
}
